{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "05ba7105",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-25T02:12:59.271067Z",
     "start_time": "2024-02-25T02:12:57.502163Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "from IPython.display import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b32c65e4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T15:14:06.419525Z",
     "start_time": "2023-08-06T15:14:06.392901Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1503, 6)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# data\n",
    "# http://d2l-data.s3-accelerate.amazonaws.com/airfoil_self_noise.dat\n",
    "data = np.genfromtxt('./airfoil_self_noise.dat', dtype=np.float32, delimiter='\\t')\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5c92ff96",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T15:14:28.242677Z",
     "start_time": "2023-08-06T15:14:28.225772Z"
    }
   },
   "outputs": [],
   "source": [
    "data = torch.from_numpy((data - data.mean(axis=0)) / data.std(axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5f75e73f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T15:19:30.210556Z",
     "start_time": "2023-08-06T15:19:30.202015Z"
    }
   },
   "outputs": [],
   "source": [
    "batch_size = 10\n",
    "n_features = data.shape[1]-1\n",
    "dataset = torch.utils.data.TensorDataset(data[:1500, :-1], data[:1500, -1])\n",
    "data_loader = torch.utils.data.DataLoader(dataset, batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "606d05a9",
   "metadata": {},
   "source": [
    "## adadelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f7190045",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T15:24:12.300268Z",
     "start_time": "2023-08-06T15:24:12.259890Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(4)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.square(torch.tensor(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5dba043f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T15:22:23.498139Z",
     "start_time": "2023-08-06T15:22:23.489911Z"
    }
   },
   "outputs": [],
   "source": [
    "def init_adadelta_states(n_features):\n",
    "    s_w, s_b = torch.zeros((n_features, 1)), torch.zeros(1)\n",
    "    delta_w, delta_b = torch.zeros((delta_w, 1)), torch.zeros(1)\n",
    "    return (s_w, s_b), (delta_w, delta_b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5bae7485",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T15:25:00.942544Z",
     "start_time": "2023-08-06T15:25:00.931307Z"
    }
   },
   "outputs": [],
   "source": [
    "def adadelta_step(params, states, hyperparams):\n",
    "    rho, eps = hyperparams['rho'], 1e-6\n",
    "    for p, (s, delta) in zip(params, states):\n",
    "        with torch.no_grad():\n",
    "            s[:] = rho * s + (1-rho)*torch.square(p.grad)\n",
    "            g = (torch.sqrt(delta + eps) / torch.sqrt(s + eps)) * p.grad\n",
    "            p[:] -= g\n",
    "            delta[:] = rho * delta + (1 - rho) * g * g\n",
    "        p.grad.data.zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "3cd7b543",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-08-06T15:31:36.882658Z",
     "start_time": "2023-08-06T15:31:36.872988Z"
    }
   },
   "outputs": [],
   "source": [
    "def train(step_fn, states, hyperparams, data_loader, n_features, num_epochs=5):\n",
    "    w = torch.normal(mean=0.0, std=0.01, size=(feature_dim, 1),\n",
    "                     requires_grad=True)\n",
    "    b = torch.zeros((1), requires_grad=True)\n",
    "    model = lambda X: torch.matmul(X, w) + b, \n",
    "    loss_fn = lambda y, y_hat: 1/2 * (y_hat - y) ** 2 \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "692a8a81",
   "metadata": {},
   "source": [
    "## Adam: Adaptive Moment Estimation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e29a601d",
   "metadata": {},
   "source": [
    "$$\n",
    "\\begin{split}\n",
    "&m_0=0,v_0=0\\\\\n",
    "&m_t=\\beta_1m_{t-1}+(1-\\beta_1)g_t\\\\\n",
    "&v_t=\\beta_2v_{t-1}+(1-\\beta_2)g_t^2\\\\\n",
    "&\\hat {m}_t=\\frac{m_t}{1-\\beta_1^t}\\\\\n",
    "&\\hat {v}_t=\\frac{v_t}{1-\\beta_2^t}\\\\\n",
    "&\\theta_{t+1}=\\theta_t-\\frac{\\eta \\hat{m}_t}{\\sqrt{\\hat{v}_t}+\\epsilon}\n",
    "\\end{split}\n",
    "$$\n",
    "\n",
    "- 自适应（adaptive）体现在每个参数（parameter）的学习率\n",
    "    - $m_t$ <= Momentum\n",
    "    - $v_t$ <= RMSprop\n",
    "- 关于 $m_t, v_t$ 的计算，可以称之为 EMA（exponential moving average）\n",
    "    - $m_t$ 是**过去所有梯度**的加权平均\n",
    "    - $m_t=\\beta_1^0(1-\\beta_1)g_t+\\beta_1^1(1-\\beta_1)g_{t-1}+\\beta_1^2(1-\\beta_1)g_{t-2}+\\cdots+\\beta_1^{t-1}(1-\\beta_1)g_1$\n",
    "    - $(1-\\beta_1)\\gt \\beta_1(1-\\beta_1)\\gt \\beta_1^2(1-\\beta_1)\\gt\\cdots\\gt \\beta_1^{t-1}(1-\\beta_1)$\n",
    "    - 越近（$t$ 越大）的梯度，对 update 的贡献更大；\n",
    "    - 分析下各个时刻 $g_t$ 前的系数 $\\beta^{t-1}(1-\\beta)$\n",
    "    \n",
    "    $$\n",
    "    \\begin{split}\n",
    "    &=\\beta^0(1-\\beta)+\\beta^1(1-\\beta)+\\beta^2(1-\\beta)+\\cdots+\\beta^{t-1}(1-\\beta)\\\\\n",
    "    &=(1-\\beta)(\\beta^0+\\beta^1+\\cdots+\\beta^{t-1})\\\\\n",
    "    &\\approx(1-\\beta)\\frac{1}{1-\\beta}\\approx1\n",
    "    \\end{split}\n",
    "    $$\n",
    "    \n",
    "- $\\hat{m}_t$ is the bias-corrected（偏差校正） first moment (mean) estimate of the gradients.\n",
    "    - $m_0=0$ => $m_t$ 在初始阶段被低估，$1/(1-\\beta_1^t)$ 初始会进行比较大的更新，但随着 $t$ 的变大，分母越开越趋近于1\n",
    "- $\\hat{v}_t$ is the bias-corrected second moment (uncentered variance) estimate of the gradients.\n",
    "- optimizer states 的角度\n",
    "    - $m_t, v_t$ 都与 $g_t$ 与模型的参数量相同；$m_t,v_t$ 称之为 optimizer states\n",
    "    - 模型的参数量为 $N$，$m, v$ 的大小分别也为 $N$；\n",
    "    - deepseed ZeRO stage\n",
    "- Adam tweaks the gradient descent method by considering the moving average of the first and second-order moments of the gradient."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "79122472",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-25T09:15:57.585568Z",
     "start_time": "2024-02-25T09:15:57.385211Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fa28ed193c0>]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAGdCAYAAADqsoKGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA9DklEQVR4nO3de3iU9Z3//9dkMpkhIQmQQEIghASUg3ggiUUQpKcNB3VL163oVyP99nu5zVaFkG2LSvtt19ZGtm6/1gpYWrS1VOTnRlbqSkuwEjkEEAiIEkU5hUNCCMIMxxw/vz+SjIQMkEkmuTOT5+O65krmvt9z3++5PeR1fe7Pfd82Y4wRAABAkAuzugEAAIBAINQAAICQQKgBAAAhgVADAABCAqEGAACEBEINAAAICYQaAAAQEgg1AAAgJIRb3UBXamho0LFjxxQdHS2bzWZ1OwAAoA2MMTpz5oySkpIUFnbl8ZgeFWqOHTum5ORkq9sAAADtcPjwYQ0ePPiK63tUqImOjpbUeFBiYmIs7gYAALSFx+NRcnKy9+/4lfSoUNN8yikmJoZQAwBAkLnW1BEmCgMAgJBAqAEAACGBUAMAAEICoQYAAIQEQg0AAAgJhBoAABASCDUAACAkEGoAAEBIINQAAICQ0K5Qs2jRIqWmpsrlcikjI0Pr16+/an1RUZEyMjLkcrmUlpamF198scX6jz76SPfcc4+GDh0qm82m5557LiD7BQAAPYffoWbFihXKzc3V/PnzVVJSokmTJmnatGkqKyvzWX/gwAFNnz5dkyZNUklJiZ588knNnj1bBQUF3prz588rLS1NzzzzjBITEwOyXwAA0LPYjDHGnw+MGzdO6enpWrx4sXfZqFGjNGPGDOXn57eqnzdvnlatWqXS0lLvspycHO3atUvFxcWt6ocOHarc3Fzl5uZ2aL++eDwexcbGyu128+wnAACCRFv/fvs1UlNTU6Pt27crKyurxfKsrCxt2rTJ52eKi4tb1U+ZMkXbtm1TbW1tp+1Xkqqrq+XxeFq8As0Yo9+886n+7f/bpTMX2/Z9AABA4PkVaqqqqlRfX6+EhIQWyxMSElRRUeHzMxUVFT7r6+rqVFVV1Wn7laT8/HzFxsZ6X8nJyW3anz9sNpv+WHxQBTuO6NDJ8wHfPgAAaJt2TRS+/NHfxpirPg7cV72v5YHe7xNPPCG32+19HT582K/9tVVKXJQk6eDJc52yfQAAcG3h/hTHx8fLbre3Gh2prKxsNYrSLDEx0Wd9eHi44uLiOm2/kuR0OuV0Otu0j45I6Rep7YdOMVIDAICF/BqpiYiIUEZGhgoLC1ssLyws1IQJE3x+Zvz48a3q16xZo8zMTDkcjk7bb1dqHqkpI9QAAGAZv0ZqJCkvL0/Z2dnKzMzU+PHjtWTJEpWVlSknJ0dS4ymfo0eP6pVXXpHUeKXTCy+8oLy8PD388MMqLi7W0qVLtXz5cu82a2pqtGfPHu/vR48e1c6dO9W7d28NHz68Tfu1UkpcpCROPwEAYCW/Q83MmTN18uRJPfXUUyovL9eYMWP09ttvKyUlRZJUXl7e4t4xqampevvttzV37lwtXLhQSUlJev7553XPPfd4a44dO6axY8d63z/77LN69tlnNXnyZK1bt65N+7XSkKZQU/Y5IzUAAFjF7/vUBLPOuk/N5+dqlP6zxlNjH/9sqlwOe8C2DQBAT9cp96mBb30jHYp2Ng56HWa0BgAASxBqAsBmsyklvnleDaEGAAArEGoCJKVf4xVQh5gsDACAJQg1AZLCZGEAACxFqAmQLy7rJtQAAGAFQk2ADOnXfAM+Tj8BAGAFQk2ADG2aKHzk1AXV1TdY3A0AAD0PoSZAEqJdiggPU12D0bHTF61uBwCAHodQEyBhYTal9ONxCQAAWIVQE0DNk4UPcQUUAABdjlATQEwWBgDAOoSaABrKXYUBALAMoSaAhjTNqSkj1AAA0OUINQE0NK7pUQmfn1MPevg5AADdAqEmgAb17SV7mE0XaxtUeaba6nYAAOhRCDUB5LCHKamPS5J0sIrJwgAAdCVCTYA1n4LiXjUAAHQtQk2ApcY3hpoDVUwWBgCgKxFqAuyLUHPW4k4AAOhZCDUB9kWo4fQTAABdiVATYGnxvSU13oCvvoHLugEA6CqEmgAb1LeXIuxhqqlr0LHTF6xuBwCAHoNQE2D2MJv3wZacggIAoOsQajoB82oAAOh6hJpOkNqfUAMAQFcj1HSCtKaRmv2EGgAAugyhphOkNl0Bxb1qAADoOoSaTtA8p+bIqQuqrqu3uBsAAHoGQk0niO8doWhnuIyRyk7yuAQAALoCoaYT2Gw272Rh5tUAANA1CDWdhMu6AQDoWoSaTuINNScINQAAdAVCTSdhpAYAgK5FqOkkzQ+2ZE4NAABdg1DTSYbGNz7/qepstTwXay3uBgCA0Eeo6STRLof6RzslSQcZrQEAoNMRajoR82oAAOg6hJpO1PwMqH2VPC4BAIDORqjpRMP6N04W3sdl3QAAdDpCTScaPqAx1HzGSA0AAJ2OUNOJmkPNgapzqm8wFncDAEBoI9R0oqQ+veQMD1NNfYMOf86DLQEA6EyEmk5kD7MprT+noAAA6AqEmk7mnVdzglADAEBnItR0suHNV0AxUgMAQKci1HQyRmoAAOgahJpONmxA4w34Pqs8K2O4AgoAgM5CqOlkqfFRCrNJZy7W6cSZaqvbAQAgZBFqOpkz3K4h/Rqf2M0pKAAAOg+hpgs0z6thsjAAAJ2HUNMFhnGvGgAAOh2hpgsM4wooAAA6HaGmC/BgSwAAOh+hpgs0n3467qnWmYu1FncDAEBoItR0gdheDvWPdkqS9p04Z3E3AACEJkJNFxnOZGEAADoVoaaLMK8GAIDO1a5Qs2jRIqWmpsrlcikjI0Pr16+/an1RUZEyMjLkcrmUlpamF198sVVNQUGBRo8eLafTqdGjR2vlypUt1tfV1elHP/qRUlNT1atXL6Wlpempp55SQ0NDe75Cl/si1JyxuBMAAEKT36FmxYoVys3N1fz581VSUqJJkyZp2rRpKisr81l/4MABTZ8+XZMmTVJJSYmefPJJzZ49WwUFBd6a4uJizZw5U9nZ2dq1a5eys7N17733asuWLd6aBQsW6MUXX9QLL7yg0tJS/cd//Id++ctf6je/+U07vnbXuz4hWpK09zgjNQAAdAab8fMpi+PGjVN6eroWL17sXTZq1CjNmDFD+fn5rernzZunVatWqbS01LssJydHu3btUnFxsSRp5syZ8ng8Wr16tbdm6tSp6tu3r5YvXy5Juuuuu5SQkKClS5d6a+655x5FRkbqT3/6U5t693g8io2NldvtVkxMjD9fu8NOnq1Wxs/XymaTPvr3KYqMCO/S/QMAEKza+vfbr5Gampoabd++XVlZWS2WZ2VladOmTT4/U1xc3Kp+ypQp2rZtm2pra69ac+k2J06cqHfeeUd79+6VJO3atUsbNmzQ9OnTr9hvdXW1PB5Pi5dV4no7Fd87QsYwrwYAgM7gV6ipqqpSfX29EhISWixPSEhQRUWFz89UVFT4rK+rq1NVVdVVay7d5rx583T//fdr5MiRcjgcGjt2rHJzc3X//fdfsd/8/HzFxsZ6X8nJyf583YBrPgX1SQXzagAACLR2TRS22Wwt3htjWi27Vv3ly6+1zRUrVmjZsmV69dVXtWPHDv3xj3/Us88+qz/+8Y9X3O8TTzwht9vtfR0+fPjaX64TfTGvhlADAECg+TWxIz4+Xna7vdWoTGVlZauRlmaJiYk+68PDwxUXF3fVmku3+YMf/ECPP/647rvvPknSjTfeqEOHDik/P1+zZs3yuW+n0ymn0+nPV+xUTBYGAKDz+DVSExERoYyMDBUWFrZYXlhYqAkTJvj8zPjx41vVr1mzRpmZmXI4HFetuXSb58+fV1hYy3btdnvQXNItSSMSGy/rZqQGAIDA8/sSnLy8PGVnZyszM1Pjx4/XkiVLVFZWppycHEmNp3yOHj2qV155RVLjlU4vvPCC8vLy9PDDD6u4uFhLly71XtUkSXPmzNEdd9yhBQsW6Bvf+IbefPNNrV27Vhs2bPDW3H333Xr66ac1ZMgQ3XDDDSopKdGvfvUrfec73+noMegy1zWN1JS7L8p9oVaxvRwWdwQAQAgx7bBw4UKTkpJiIiIiTHp6uikqKvKumzVrlpk8eXKL+nXr1pmxY8eaiIgIM3ToULN48eJW23z99dfNiBEjjMPhMCNHjjQFBQUt1ns8HjNnzhwzZMgQ43K5TFpampk/f76prq5uc99ut9tIMm63278vHEC3/WKtSZn3lnn/wEnLegAAIJi09e+33/epCWZW3qem2ayXtqpo7wn94ps36n+NG2JJDwAABJNOuU8NOm5EIldAAQDQGQg1XYx71QAA0DkINV3s+gSugAIAoDMQarrY8AG9ZbNJJ8/VqOpstdXtAAAQMgg1XSwyIlxD+kVKYrQGAIBAItRYwHtnYebVAAAQMIQaC3jn1fC0bgAAAoZQYwGugAIAIPAINRYYmdh446BPKs6ooaHH3PsQAIBORaixQFr/KEXYw3S2uk5HTl2wuh0AAEICocYCDnuYrmuaV7On3GNxNwAAhAZCjUVGDWw8BVVKqAEAICAINRYh1AAAEFiEGouMGth4BVRpBaEGAIBAINRYZHTTSM3hzy/ozMVai7sBACD4EWos0icyQgNjXZKkj7lfDQAAHUaosRDzagAACBxCjYW882oINQAAdBihxkLNIzV7yjn9BABARxFqLNQcaj6p8KiexyUAANAhhBoLDY2LkssRpou1DTp48pzV7QAAENQINRayh9k0IpHJwgAABAKhxmKjmSwMAEBAEGos9sVl3UwWBgCgIwg1FuNeNQAABAahxmIjExtPP5W7L+rzczUWdwMAQPAi1Fgs2uVQanyUJOnDo26LuwEAIHgRarqBG5IaT0F9eIxQAwBAexFquoEbB8VKYqQGAICOINR0A2O8oYbJwgAAtBehphtoPv1U9vl5uc/XWtwNAADBiVDTDfSJjFByv16SpI+YVwMAQLsQarqJMUlNp6AINQAAtAuhpptgXg0AAB1DqOkmxnAFFAAAHUKo6SaaJwvvrzqnMxeZLAwAgL8INd1EfG+nBsa6JPFwSwAA2oNQ0400n4LazSkoAAD8RqjpRpqvgPqIUAMAgN8INd3ImEGN82oYqQEAwH+Emm6k+RlQ+06c1fmaOou7AQAguBBqupEBMS71j3aqwUh7jnG/GgAA/EGo6WZuHtxHkrTz8GlL+wAAINgQarqZsUP6SJJ2HWFeDQAA/iDUdDPNIzW7GKkBAMAvhJpu5sbBjZOFyz4/r8/P1VjcDQAAwYNQ083E9nIorX+UJGnXkdPWNgMAQBAh1HRDt3AKCgAAvxFquqGbk/tIItQAAOAPQk035A01R9wyxljbDAAAQYJQ0w2NGhgth92mz8/V6MipC1a3AwBAUCDUdEPOcLtGD2x8DhQ34QMAoG0INd0U82oAAPAPoaab8t6Ej8u6AQBoE0JNN9U8UrP7qFt19Q3WNgMAQBAg1HRTafFRinaG62Jtgz45fsbqdgAA6PbaFWoWLVqk1NRUuVwuZWRkaP369VetLyoqUkZGhlwul9LS0vTiiy+2qikoKNDo0aPldDo1evRorVy5slXN0aNH9eCDDyouLk6RkZG65ZZbtH379vZ8hW4vLMzmHa0pKTttaS8AAAQDv0PNihUrlJubq/nz56ukpESTJk3StGnTVFZW5rP+wIEDmj59uiZNmqSSkhI9+eSTmj17tgoKCrw1xcXFmjlzprKzs7Vr1y5lZ2fr3nvv1ZYtW7w1p06d0u233y6Hw6HVq1drz549+s///E/16dPH/28dJNJT+kqSdhw6ZXEnAAB0fzbj593dxo0bp/T0dC1evNi7bNSoUZoxY4by8/Nb1c+bN0+rVq1SaWmpd1lOTo527dql4uJiSdLMmTPl8Xi0evVqb83UqVPVt29fLV++XJL0+OOPa+PGjdccFboaj8ej2NhYud1uxcTEtHs7XaVo7wnNemmrUuIiVfSDr1jdDgAAlmjr32+/Rmpqamq0fft2ZWVltVielZWlTZs2+fxMcXFxq/opU6Zo27Ztqq2tvWrNpdtctWqVMjMz9a1vfUsDBgzQ2LFj9bvf/e6q/VZXV8vj8bR4BZNbkvvIZpMOnTyvE2eqrW4HAIBuza9QU1VVpfr6eiUkJLRYnpCQoIqKCp+fqaio8FlfV1enqqqqq9Zcus39+/dr8eLFuu666/S3v/1NOTk5mj17tl555ZUr9pufn6/Y2FjvKzk52Z+va7nYXg5dPyBakrSjjFNQAABcTbsmCttsthbvjTGtll2r/vLl19pmQ0OD0tPT9Ytf/EJjx47Vd7/7XT388MMtToNd7oknnpDb7fa+Dh8+fO0v180wrwYAgLbxK9TEx8fLbre3GpWprKxsNdLSLDEx0Wd9eHi44uLirlpz6TYHDhyo0aNHt6gZNWrUFScoS5LT6VRMTEyLV7DJaAo12wk1AABclV+hJiIiQhkZGSosLGyxvLCwUBMmTPD5mfHjx7eqX7NmjTIzM+VwOK5ac+k2b7/9dn3yySctavbu3auUlBR/vkLQaQ41Hxx1q7qu3uJuAADovvw+/ZSXl6ff//73eumll1RaWqq5c+eqrKxMOTk5khpP+Tz00EPe+pycHB06dEh5eXkqLS3VSy+9pKVLl+r73/++t2bOnDlas2aNFixYoI8//lgLFizQ2rVrlZub662ZO3euNm/erF/84hf67LPP9Oqrr2rJkiV65JFHOvD1u7+hcZHqFxWhmroGfXQsuCY6AwDQpUw7LFy40KSkpJiIiAiTnp5uioqKvOtmzZplJk+e3KJ+3bp1ZuzYsSYiIsIMHTrULF68uNU2X3/9dTNixAjjcDjMyJEjTUFBQauav/zlL2bMmDHG6XSakSNHmiVLlvjVt9vtNpKM2+3263NW+z9/eN+kzHvL/O69fVa3AgBAl2vr32+/71MTzILtPjXNFq/bpwV//VjTxiRq8YMZVrcDAECX6pT71MAazfNqth06pR6UQQEA8AuhJgjcNDhW4WE2nThTrSOnLljdDgAA3RKhJgi4HHbdMChWEpd2AwBwJYSaIJHpPQX1ucWdAADQPRFqgsSXUvtJkrbsJ9QAAOALoSZI3Dq0MdR8WnlWJ8/ycEsAAC5HqAkS/aIidH1Cb0nS+weZVwMAwOUINUGk+RTU1gOcggIA4HKEmiDypdTGB4BuOXDS4k4AAOh+CDVBZFzTSM2eco88F2st7gYAgO6FUBNEEmJcGhoXKWOk7cyrAQCgBUJNkPFe2s28GgAAWiDUBJnmeTVbmVcDAEALhJog0zyv5oMjbp2vqbO4GwAAug9CTZAZ3LeXBsa6VNdgVFJ22up2AADoNgg1QcZms13yyAROQQEA0IxQE4TGNc2r2cxzoAAA8CLUBKHbhzeGmpLDp5hXAwBAE0JNEBrSL1KD+vRSbb3hkQkAADQh1AQhm82mCcMaR2uK9zGvBgAAiVATtG4fHi9J2rivyuJOAADoHgg1QWp800jNR8c8On2+xuJuAACwHqEmSCXEuDR8QG8ZI23m0m4AAAg1wez2ptGajZ8RagAAINQEsQnMqwEAwItQE8RuS41TmE3af+KcKtwXrW4HAABLEWqCWGykQ2MGxUqSNjFaAwDo4Qg1QW7CsKZTUMyrAQD0cISaIDexaV7Nhs9OyBhjcTcAAFiHUBPkMof2lcsRpuOean1y/IzV7QAAYBlCTZBzOey6La3x0u739p6wuBsAAKxDqAkBk6/vL0kqItQAAHowQk0IaA417x84pfM1dRZ3AwCANQg1ISA1PkqD+/ZSTX0Dj0wAAPRYhJoQYLPZvjgF9QmnoAAAPROhJkQwrwYA0NMRakLE+GFxCg+z6eDJ8zp08pzV7QAA0OUINSEi2uVQRkpfSVzaDQDomQg1IWTyCE5BAQB6LkJNCGmeV7Np30ldrK23uBsAALoWoSaEjB4Yo4QYp87X1HNpNwCgxyHUhBCbzaavjkyQJL1TWmlxNwAAdC1CTYj5+qgBkqS/f1zJU7sBAD0KoSbE3D48Xi5HmI6evqCPK3hqNwCg5yDUhBiXw66Jw+MlSe+UHre4GwAAug6hJgR9bVTjvJq1zKsBAPQghJoQ9NWRjfNqdh05rRNnqi3uBgCArkGoCUEJMS7dOChWxkjvfsJoDQCgZyDUhKivNV0FxbwaAEBPQagJUV9vmlez/tMq7i4MAOgRCDUh6oakGA2Mdel8Tb02flZldTsAAHQ6Qk2IstlsmnJDoiTprx9WWNwNAACdj1ATwppDTWHpcdXWN1jcDQAAnYtQE8K+lNpPcVEROn2+VlsPfG51OwAAdCpCTQizh9n0D6MbJwxzCgoAEOoINSFuypjGU1B/+6hCDQ084BIAELraFWoWLVqk1NRUuVwuZWRkaP369VetLyoqUkZGhlwul9LS0vTiiy+2qikoKNDo0aPldDo1evRorVy58orby8/Pl81mU25ubnva71FuHxavaGe4Ks9Uq+TwKavbAQCg0/gdalasWKHc3FzNnz9fJSUlmjRpkqZNm6aysjKf9QcOHND06dM1adIklZSU6Mknn9Ts2bNVUFDgrSkuLtbMmTOVnZ2tXbt2KTs7W/fee6+2bNnSanvvv/++lixZoptuusnf1nukiPAw7434Vu/mFBQAIHTZjDF+nZMYN26c0tPTtXjxYu+yUaNGacaMGcrPz29VP2/ePK1atUqlpaXeZTk5Odq1a5eKi4slSTNnzpTH49Hq1au9NVOnTlXfvn21fPly77KzZ88qPT1dixYt0s9//nPdcssteu6559rcu8fjUWxsrNxut2JiYvz52kHtrx+WK2fZDg3u20vrf/gV2Ww2q1sCAKDN2vr326+RmpqaGm3fvl1ZWVktlmdlZWnTpk0+P1NcXNyqfsqUKdq2bZtqa2uvWnP5Nh955BHdeeed+vrXv96mfqurq+XxeFq8eqLJ1w+QyxGmI6cu6MOjPfMYAABCn1+hpqqqSvX19UpISGixPCEhQRUVvk9tVFRU+Kyvq6tTVVXVVWsu3eZrr72mHTt2+BwNupL8/HzFxsZ6X8nJyW3+bCjpFWH3Prn7rQ+OWdwNAACdo10ThS8/fWGMueopDV/1ly+/2jYPHz6sOXPmaNmyZXK5XG3u84knnpDb7fa+Dh8+3ObPhpq7b0qSJL31QTlXQQEAQlK4P8Xx8fGy2+2tRmUqKytbjbQ0S0xM9FkfHh6uuLi4q9Y0b3P79u2qrKxURkaGd319fb3ee+89vfDCC6qurpbdbm+1b6fTKafT6c9XDFlfGTlAvZ3hOnr6gnaUnVLm0H5WtwQAQED5NVITERGhjIwMFRYWtlheWFioCRMm+PzM+PHjW9WvWbNGmZmZcjgcV61p3ubXvvY17d69Wzt37vS+MjMz9cADD2jnzp0+Aw1acjnsymq6Ed9fdnEKCgAQevwaqZGkvLw8ZWdnKzMzU+PHj9eSJUtUVlamnJwcSY2nfI4ePapXXnlFUuOVTi+88ILy8vL08MMPq7i4WEuXLm1xVdOcOXN0xx13aMGCBfrGN76hN998U2vXrtWGDRskSdHR0RozZkyLPqKiohQXF9dqOa7s7luS9EbJUf3P7nL9+K7RCrdz70UAQOjwO9TMnDlTJ0+e1FNPPaXy8nKNGTNGb7/9tlJSUiRJ5eXlLe5Zk5qaqrfffltz587VwoULlZSUpOeff1733HOPt2bChAl67bXX9KMf/Ug//vGPNWzYMK1YsULjxo0LwFdEs4nD49U30qGqszXavP9zTbwu3uqWAAAIGL/vUxPMeup9ai715MrdenVLmWZmJmvBP3MDQwBA99cp96lB8Gu+Cmr1h+Wqrqu3uBsAAAKHUNPDfCm1nxJinPJcrNN7e6usbgcAgIAh1PQw9jCb7moarVlZcsTibgAACBxCTQ90T/pgSdLaPZU6fb7G4m4AAAgMQk0PNDopRqMGxqimvkF/+aDc6nYAAAgIQk0PdU/6IElSwXZOQQEAQgOhpof6xi2DZA+zaefh09p34qzV7QAA0GGEmh6qf7RTk6/vL4nRGgBAaCDU9GDNE4ZXlhxVPU/uBgAEOUJND/a1UQMU4wpXufuiivedtLodAAA6hFDTg7kcdt19c+M9a17fftjibgAA6BhCTQ8389ZkSdLqDyt06hz3rAEABC9CTQ9346BY3ZAUo5q6Br1RctTqdgAAaDdCTQ9ns9l0/5eGSJKWby1TD3poOwAgxBBqoG/ckqReDrs+qzyrbYdOWd0OAADtQqiBol0O3X3zQEnS8i1lFncDAED7EGogSd5TUG/tLuchlwCAoESogSTpluQ+GpkYrZq6Bq1kwjAAIAgRaiCpccLw/xrXOFqzbPMhJgwDAIIOoQZe3xw7SFERdu07cU4bP+MOwwCA4EKogVe0y6F7MhqfB/WHTQetbQYAAD8RatDCQ+OHSpLe+fi4yk6et7YZAAD8QKhBC8MH9Nak6+JljPSnzQetbgcAgDYj1KCVb08YKkla8f5hna+ps7YZAADaiFCDVr48YoCG9IuU52Kd/rvkmNXtAADQJoQatGIPs+mh8SmSpD9sOsDl3QCAoECogU/fykxWVIRde4+f1bq9J6xuBwCAayLUwKfYXg7voxOWFO23uBsAAK6NUIMr+s7EVIWH2VS8/6Q+OHLa6nYAALgqQg2uKKlPL919c5Ik6bfvMVoDAOjeCDW4qocnpUmSVu8u52Z8AIBujVCDqxqdFKNJ18WrwUhLNzBaAwDovgg1uKbv3jFMkrRi22GdPFttcTcAAPhGqME13T48TjcNjtXF2gb9fsMBq9sBAMAnQg2uyWaz6bGvXidJemXTQZ06V2NxRwAAtEaoQZt8fdQAjRoYo3M19Xp5I6M1AIDuh1CDNrHZbJr91eGSpJc3HpT7Qq3FHQEA0BKhBm025YZEXZ/QW2eq6/SHjQetbgcAgBYINWizsLAv5ta8tPGAzlxktAYA0H0QauCX6TcO1LD+UXJfqNXv1zO3BgDQfRBq4Bd7mE3/ljVCkvT79fu5bw0AoNsg1MBv08Yk6sZBsTpXU6+F7+6zuh0AACQRatAONptNP5zaOFqzbPMhHTnFM6EAANYj1KBdJg6P1/i0ONXUN+jXaz+1uh0AAAg1aJ9LR2sKdhzRp8fPWNwRAKCnI9Sg3cYO6aus0QlqMNIv//aJ1e0AAHo4Qg065AdTRijMJq3Zc1zF+05a3Q4AoAcj1KBDrkuI1gPjUiRJT721R/UNxuKOAAA9FaEGHTb3H65XjCtcpeUerXj/sNXtAAB6KEINOqxfVIRyv369JOk/13wiD49PAABYgFCDgMgen6Jh/aN08lyNXvj7Z1a3AwDogQg1CAiHPUw/umu0JOnljQd0oOqcxR0BAHoaQg0C5isjBujLI/qrtt7o/775oYxh0jAAoOsQahBQP737BkWEh2n9p1VateuY1e0AAHoQQg0Camh8lB77ynBJ0s/eKpX7ApOGAQBdg1CDgPuXyWka1j9KVWer9R9//djqdgAAPUS7Qs2iRYuUmpoql8uljIwMrV+//qr1RUVFysjIkMvlUlpaml588cVWNQUFBRo9erScTqdGjx6tlStXtlifn5+vW2+9VdHR0RowYIBmzJihTz7h1vzdkTPcrqe/eaMk6c9byrT90CmLOwIA9AR+h5oVK1YoNzdX8+fPV0lJiSZNmqRp06aprKzMZ/2BAwc0ffp0TZo0SSUlJXryySc1e/ZsFRQUeGuKi4s1c+ZMZWdna9euXcrOzta9996rLVu2eGuKior0yCOPaPPmzSosLFRdXZ2ysrJ07hxX2XRHt6XF6VsZgyVJ81fuVk1dg8UdAQBCnc34eYnKuHHjlJ6ersWLF3uXjRo1SjNmzFB+fn6r+nnz5mnVqlUqLS31LsvJydGuXbtUXFwsSZo5c6Y8Ho9Wr17trZk6dar69u2r5cuX++zjxIkTGjBggIqKinTHHXe0qXePx6PY2Fi53W7FxMS06TNov8/P1ejrvyrS5+dqNPtr1ynvH663uiUAQBBq699vv0ZqampqtH37dmVlZbVYnpWVpU2bNvn8THFxcav6KVOmaNu2baqtrb1qzZW2KUlut1uS1K9fvyvWVFdXy+PxtHih6/SLitDPvjFGkrTw3c+0+4jb4o4AAKHMr1BTVVWl+vp6JSQktFiekJCgiooKn5+pqKjwWV9XV6eqqqqr1lxpm8YY5eXlaeLEiRozZswV+83Pz1dsbKz3lZycfM3viMC686aBuvOmgapvMPq313equq7e6pYAACGqXROFbTZbi/fGmFbLrlV/+XJ/tvnoo4/qgw8+uOKpqWZPPPGE3G6393X4MA9btMLPvjFG8b0jtPf4WT239lOr2wEAhCi/Qk18fLzsdnurEZTKyspWIy3NEhMTfdaHh4crLi7uqjW+tvnYY49p1apVevfddzV48OCr9ut0OhUTE9Piha7XLyrCezXUb4v2aUcZV0MBAALPr1ATERGhjIwMFRYWtlheWFioCRMm+PzM+PHjW9WvWbNGmZmZcjgcV625dJvGGD366KN644039Pe//12pqan+tA6LTbkhUd8cO0gNRpq7YqfO8CRvAECA+X36KS8vT7///e/10ksvqbS0VHPnzlVZWZlycnIkNZ7yeeihh7z1OTk5OnTokPLy8lRaWqqXXnpJS5cu1fe//31vzZw5c7RmzRotWLBAH3/8sRYsWKC1a9cqNzfXW/PII49o2bJlevXVVxUdHa2KigpVVFTowoULHfj66Eo/vfsGDerTS4dOnteP/5tnQwEAAsy0w8KFC01KSoqJiIgw6enppqioyLtu1qxZZvLkyS3q161bZ8aOHWsiIiLM0KFDzeLFi1tt8/XXXzcjRowwDofDjBw50hQUFLRYL8nn6+WXX25z326320gybrfbr++LwNl28KRJe+J/TMq8t8zr2w5b3Q4AIAi09e+33/epCWbcp6Z7eOHvn+rZNXsVGWHXW49NVFr/3la3BADoxjrlPjVAIPzrl4drfFqcztfU69FXS3Sxlsu8AQAdR6hBl7OH2fT/Zt6ivpEO7Sn36KerPrK6JQBACCDUwBKJsS79+r6xstmk194/rOVbfT87DACAtiLUwDJ3XN9f388aIUn6yZsfqYT71wAAOoBQA0t978vDNOWGBNXUN+hfl+3QiTPVVrcEAAhShBpYymaz6dlv3axh/aNU4bmo7/15O8+HAgC0C6EGlot2ObTkoUxFO8P1/sFTerxgNzfmAwD4jVCDbmFY/95a9GC67GE2rSw5ql+/w4MvAQD+IdSg25h0XX/9fMYYSdJzaz/VypIjFncEAAgmhBp0K/d/aYi+OzlNkjTvv3areN9JizsCAAQLQg26nXlTRmramETV1Dfo4Ve2afcRt9UtAQCCAKEG3U5Y0x2Hb0vrp7PVdZr18lZ9VnnW6rYAAN0coQbdksth1+8eytSNg2L1+bkaPbR0i46evmB1WwCAboxQg24r2uXQH/73rRrWP0rH3BeV/fstqvRctLotAEA3RahBtxbX26k//Z9xGtSnl/ZXndN9SzbrOMEGAOADoQbdXlKfXnrtX25rEWwq3AQbAEBLhBoEheR+kd5gc6DqnO5bUqxyN3NsAABfINQgaCT3i9SK796mwX176eDJ8/rnxcXaf4KrogAAjQg1CCqD+0ZqxXfHKzU+SkdPX9A/v1isD46ctrotAEA3QKhB0BnUp5dezxnvvdz7/iWbteHTKqvbAgBYjFCDoBTf26nl/3Kbbh8ep3M19frff9iqN3cetbotAICFCDUIWr2d4Xrp27fqzpsGqrbeaM5rO/Wrwr1qaDBWtwYAsAChBkHNGW7X8/eN1b/c0fgQzOff+VSPLt+hCzX1FncGAOhqhBoEPXuYTU9OH6X/+Oeb5LDb9PbuCn3rt5u45BsAehhCDULGvZnJevXh29QvKkIfHvXo7t9s0MbPmEAMAD0FoQYh5dah/fTmI7dr1MAYVZ2t0YNLt+j5dz5lng0A9ACEGoSc5H6RWvm9Cbrv1mQZI/2qcK9mvbxVJ89WW90aAKATEWoQklwOu5655yY9+62b5XKEaf2nVZr26/Uq2nvC6tYAAJ2EUIOQ9s8Zg/XmIxM1fEBvVZ6p1qyXtur/vvkhV0cBQAgi1CDkjUiM1luPTdS3JwyVJL1SfEh3Pr9eOw+ftrQvAEBgEWrQI7gcdv30H2/QK9/5khJinNpfdU7/tGijnvrLHp2rrrO6PQBAABBq0KPccX1//S33Dn3jliQ1GOmljQeU9f/e098/Pm51awCADiLUoMfpExmhX983Vn/437dqcN9eOnr6gr7zh2165M87VOG+aHV7AIB2ItSgx/ryiAFaM/cOffeONNnDbPqf3eX6yrPr9Ou1nzKRGACCkM0Y02PuSubxeBQbGyu3262YmBir20E38tExt37y5kfaduiUJGlgrEuPTxupf7w5STabzeLuAKBna+vfb0IN0MQYo7c+KNczqz/W0dONz426ObmPfpA1QrcPjyPcAIBFCDU+EGrQFhdr6/X79fu1aN0+nW86DTUutZ++P2WEbh3az+LuAKDnIdT4QKiBPyrPXNSid/fp1S1lqqlvkCRNvr6/HvvqcGUSbgCgyxBqfCDUoD2Onb6g3/z9M72+7bDqmh6MmZnSVzmTh+mrIwcoLIzTUgDQmQg1PhBq0BGHTp7T4nX79MaOo96Rm+sG9NZ3Jw/T3TcPlDPcbnGHABCaCDU+EGoQCMc9F/XSxgN6dXOZzjTdjTguKkIzb03WA7elaFCfXhZ3CAChhVDjA6EGgeS5WKs/by7THzcdVIWn8aZ9YTbpa6MSlH1biiYOj+fUFAAEAKHGB0INOkNdfYPWlh7XnzYf0sbPTnqXJ8W69M30QbonfbDS+ve2sEMACG6EGh8INehsn1We1bLNh/TGjiPyXPziQZljh/TRPemDdddNA9UnMsLCDgEg+BBqfCDUoKtcrK3X2tLjKth+RO99WqX6pqumwsNsmjA8XtPHJCrrhkT1iyLgAMC1EGp8INTACpVnLmrVzmMq2HFUpeUe73J7mE23pfXT1DED9dWRA5hgDABXQKjxgVADq+0/cVarP6zQ6g/L9eFRT4t1IxKi9eWR/fWVEQOUkdJXDjvPmwUAiVDjE6EG3UnZyfN6+8Nyrd1zXDvKTqnhkv8So13hmjg8XuOHxWl8WpyGD+jNs6cA9FiEGh8INeiuTp+v0XufVundjytVtPeEPj9X02J9fG+nbkvrp/HD4nRbWpzS4qMIOQB6DEKND4QaBIP6BqMPjpzWxs+qVLz/pLYdPKXquoYWNX0jHboluY/GDumrsUP66ObkPopxOSzqGAA6F6HGB0INglF1Xb12lp1W8f6T2rz/pHaUnVbNZSHHZpOG9++tm5P76IakGI0eGKNRSTEEHQAhgVDjA6EGoaCmrkGl5R6VlJ1SyeHTKik7rbLPz/usTe7XS6MHxmj0wFiNGhit6xKildy3l8KZhAwgiBBqfCDUIFRVna1WSdlp7T7q1p5jHpWWe3T09AWftQ67TUPjojR8QG8N699bwwZEaXj/aKX1j1KUM7yLOweAayPU+ECoQU9y+nyN9pR7tOeYR3vKPfq4/Iz2V53VxdqGK34mvrdTQ/r10pB+kUpuejX/nhjjkp1nWQGwAKHGB0INerqGBqOjpy9o34mz2nfinD6rPKt9J85q/4mzqjpbc9XPOuw2JfXppYQYlxJjXBoY61JC88/YxmUDop2c2gIQcG39+81YM9CDhIXZvCMwXx7Rcp37fK0Onzqvss8bX4ebfh45dUFHTp1Xbb3RoZPndeik7/k7UuNTyuN7OxXf26m43hGNP6MiFOd9H6G4qC/WuRz2Tv7GAHqSdoWaRYsW6Ze//KXKy8t1ww036LnnntOkSZOuWF9UVKS8vDx99NFHSkpK0g9/+EPl5OS0qCkoKNCPf/xj7du3T8OGDdPTTz+tb37zmx3aL4C2i410KDYyVmMGxbZaV99gVOG5qGOnL6jcfVHH3RdV4bmoikt+HvdcVF2DUeWZalWeqW7TPiMj7Irt5VBsL4dimn5e6dW8PtoVrihnuCIddoVxOgzAJfwONStWrFBubq4WLVqk22+/Xb/97W81bdo07dmzR0OGDGlVf+DAAU2fPl0PP/ywli1bpo0bN+p73/ue+vfvr3vuuUeSVFxcrJkzZ+pnP/uZvvnNb2rlypW69957tWHDBo0bN65d+wUQOPYwmwb16XXV51M1NBhVnavWcXe1qs5V6+TZGp08W62T52pUdbbx/efnGpdVna1RTX2DztfU63xNvcrdF9vVV2SEXVHOcPV2hrf6vbezMfxEOcMVFWFXrwi7XOF2OR1hcjnsja/wS35vXt5U4wwP4waHQJDxe07NuHHjlJ6ersWLF3uXjRo1SjNmzFB+fn6r+nnz5mnVqlUqLS31LsvJydGuXbtUXFwsSZo5c6Y8Ho9Wr17trZk6dar69u2r5cuXt2u/vjCnBugejDE6W12nk2dr5L5Q2+rlaf558bJ152t1rqbe+9TzzmSzSa7wL8JORHiYHPbGV0R4mCLsNu/7xmWXvbfbWn3GcclnwsNssofZFG63KcxmU3hYmOzNy5p+Xvo+7LLljb+HyW6zyW6/ZN0l78NsNtlsUpjN1vQSQQ1BqVPm1NTU1Gj79u16/PHHWyzPysrSpk2bfH6muLhYWVlZLZZNmTJFS5cuVW1trRwOh4qLizV37txWNc8991y79ytJ1dXVqq7+Yhjc4/FcsRZA17HZbIp2ORTdjpsDGmNUXdegc9V1Olddr7PVdTpXU+d9f+6S92cveX+xtl4Xaxuafjb9Xlev6kuWXait9z6DyxjpQtMyqTawB8BiYZcEnS9Cj754H9YyBF1aL0lhYVf7/KX1jZ+32SSbmn5v6qFxWdM77/rmt02fuey9vJ+1XVbfetu65DPNtc0VzdtuvX/bZfUt96fL+7vC8fWVG20+qn3W+dxoGz8b6F58bu/aofjfsq5v13/bgeBXqKmqqlJ9fb0SEhJaLE9ISFBFRYXPz1RUVPisr6urU1VVlQYOHHjFmuZttme/kpSfn69///d/b/P3A9D92Ww27ymjuN6B3bYxRrX1RhfrGkPOF4GnQTX19aqpM6qtb/C+auqNauu+eF9d16DaenPJ+gbV1l32vt6opq5xtKm+wajukp8Nl7z/Yn1D4+/GqL6+qc401TW9rzdf1LdFg5EajJHUYy5+RRf63leGBUeoaXZ5UjPGXDW9+aq/fHlbtunvfp944gnl5eV533s8HiUnJ1+xHkDPZrPZFBHeeNooGB8xYYxRg5HqGhpUV98Yfowk09AYYhqa1jfXNS8z3t/lfX/tmqbfG77YppGPmqZ9GzWOfjV12rgPb9+SaXrXvPzSmRGt1rf4bNMa492yz20b7/rGhS3WX/7+Wvtv8V0uOf5XCIltneThazaI7/20bR+++mnr9nwVtnW/kRHWXVjt157j4+Nlt9tbjY5UVla2GkVplpiY6LM+PDxccXFxV61p3mZ79itJTqdTTqezbV8OAIKczWaT3SbZw+zi5tDoify6S1ZERIQyMjJUWFjYYnlhYaEmTJjg8zPjx49vVb9mzRplZmbK4XBctaZ5m+3ZLwAA6GGMn1577TXjcDjM0qVLzZ49e0xubq6JiooyBw8eNMYY8/jjj5vs7Gxv/f79+01kZKSZO3eu2bNnj1m6dKlxOBzmv/7rv7w1GzduNHa73TzzzDOmtLTUPPPMMyY8PNxs3ry5zfttC7fbbSQZt9vt79cGAAAWaevfb79DjTHGLFy40KSkpJiIiAiTnp5uioqKvOtmzZplJk+e3KJ+3bp1ZuzYsSYiIsIMHTrULF68uNU2X3/9dTNixAjjcDjMyJEjTUFBgV/7bQtCDQAAwaetf7959hMAAOjW2vr3myfPAQCAkECoAQAAIYFQAwAAQgKhBgAAhARCDQAACAmEGgAAEBIINQAAICQQagAAQEgg1AAAgJDQo57j2nzzZI/HY3EnAACgrZr/bl/rIQg9KtScOXNGkpScnGxxJwAAwF9nzpxRbGzsFdf3qGc/NTQ06NixY4qOjpbNZgvYdj0ej5KTk3X48GGeKdWJOM5dh2PdNTjOXYPj3HU661gbY3TmzBklJSUpLOzKM2d61EhNWFiYBg8e3Gnbj4mJ4T+YLsBx7joc667Bce4aHOeu0xnH+mojNM2YKAwAAEICoQYAAIQEQk0AOJ1O/eQnP5HT6bS6lZDGce46HOuuwXHuGhznrmP1se5RE4UBAEDoYqQGAACEBEINAAAICYQaAAAQEgg1AAAgJBBqAmDRokVKTU2Vy+VSRkaG1q9fb3VLQSM/P1+33nqroqOjNWDAAM2YMUOffPJJixpjjH76058qKSlJvXr10pe//GV99NFHLWqqq6v12GOPKT4+XlFRUfrHf/xHHTlypCu/SlDJz8+XzWZTbm6udxnHOXCOHj2qBx98UHFxcYqMjNQtt9yi7du3e9dzrDuurq5OP/rRj5SamqpevXopLS1NTz31lBoaGrw1HGf/vffee7r77ruVlJQkm82m//7v/26xPlDH9NSpU8rOzlZsbKxiY2OVnZ2t06dPd/wLGHTIa6+9ZhwOh/nd735n9uzZY+bMmWOioqLMoUOHrG4tKEyZMsW8/PLL5sMPPzQ7d+40d955pxkyZIg5e/ast+aZZ54x0dHRpqCgwOzevdvMnDnTDBw40Hg8Hm9NTk6OGTRokCksLDQ7duwwX/nKV8zNN99s6urqrPha3drWrVvN0KFDzU033WTmzJnjXc5xDozPP//cpKSkmG9/+9tmy5Yt5sCBA2bt2rXms88+89ZwrDvu5z//uYmLizNvvfWWOXDggHn99ddN7969zXPPPeet4Tj77+233zbz5883BQUFRpJZuXJli/WBOqZTp041Y8aMMZs2bTKbNm0yY8aMMXfddVeH+yfUdNCXvvQlk5OT02LZyJEjzeOPP25RR8GtsrLSSDJFRUXGGGMaGhpMYmKieeaZZ7w1Fy9eNLGxsebFF180xhhz+vRp43A4zGuvveatOXr0qAkLCzN//etfu/YLdHNnzpwx1113nSksLDSTJ0/2hhqOc+DMmzfPTJw48YrrOdaBceedd5rvfOc7LZb90z/9k3nwwQeNMRznQLg81ATqmO7Zs8dIMps3b/bWFBcXG0nm448/7lDPnH7qgJqaGm3fvl1ZWVktlmdlZWnTpk0WdRXc3G63JKlfv36SpAMHDqiioqLFMXY6nZo8ebL3GG/fvl21tbUtapKSkjRmzBj+OVzmkUce0Z133qmvf/3rLZZznANn1apVyszM1Le+9S0NGDBAY8eO1e9+9zvveo51YEycOFHvvPOO9u7dK0natWuXNmzYoOnTp0viOHeGQB3T4uJixcbGaty4cd6a2267TbGxsR0+7j3qgZaBVlVVpfr6eiUkJLRYnpCQoIqKCou6Cl7GGOXl5WnixIkaM2aMJHmPo69jfOjQIW9NRESE+vbt26qGfw5feO2117Rjxw69//77rdZxnANn//79Wrx4sfLy8vTkk09q69atmj17tpxOpx566CGOdYDMmzdPbrdbI0eOlN1uV319vZ5++mndf//9kvh3ujME6phWVFRowIABrbY/YMCADh93Qk0A2Gy2Fu+NMa2W4doeffRRffDBB9qwYUOrde05xvxz+MLhw4c1Z84crVmzRi6X64p1HOeOa2hoUGZmpn7xi19IksaOHauPPvpIixcv1kMPPeSt41h3zIoVK7Rs2TK9+uqruuGGG7Rz507l5uYqKSlJs2bN8tZxnAMvEMfUV30gjjunnzogPj5edru9VbKsrKxslWRxdY899phWrVqld999V4MHD/YuT0xMlKSrHuPExETV1NTo1KlTV6zp6bZv367KykplZGQoPDxc4eHhKioq0vPPP6/w8HDvceI4d9zAgQM1evToFstGjRqlsrIySfw7HSg/+MEP9Pjjj+u+++7TjTfeqOzsbM2dO1f5+fmSOM6dIVDHNDExUcePH2+1/RMnTnT4uBNqOiAiIkIZGRkqLCxssbywsFATJkywqKvgYozRo48+qjfeeEN///vflZqa2mJ9amqqEhMTWxzjmpoaFRUVeY9xRkaGHA5Hi5ry8nJ9+OGH/HNo8rWvfU27d+/Wzp07va/MzEw98MAD2rlzp9LS0jjOAXL77be3ui3B3r17lZKSIol/pwPl/PnzCgtr+SfMbrd7L+nmOAdeoI7p+PHj5Xa7tXXrVm/Nli1b5Ha7O37cOzTNGN5LupcuXWr27NljcnNzTVRUlDl48KDVrQWFf/3XfzWxsbFm3bp1pry83Ps6f/68t+aZZ54xsbGx5o033jC7d+82999/v89LCAcPHmzWrl1rduzYYb761a/26Msy2+LSq5+M4TgHytatW014eLh5+umnzaeffmr+/Oc/m8jISLNs2TJvDce642bNmmUGDRrkvaT7jTfeMPHx8eaHP/yht4bj7L8zZ86YkpISU1JSYiSZX/3qV6akpMR7m5JAHdOpU6eam266yRQXF5vi4mJz4403ckl3d7Fw4UKTkpJiIiIiTHp6uvdyZFybJJ+vl19+2VvT0NBgfvKTn5jExETjdDrNHXfcYXbv3t1iOxcuXDCPPvqo6devn+nVq5e56667TFlZWRd/m+ByeajhOAfOX/7yFzNmzBjjdDrNyJEjzZIlS1qs51h3nMfjMXPmzDFDhgwxLpfLpKWlmfnz55vq6mpvDcfZf++++67P/yfPmjXLGBO4Y3ry5EnzwAMPmOjoaBMdHW0eeOABc+rUqQ73bzPGmI6N9QAAAFiPOTUAACAkEGoAAEBIINQAAICQQKgBAAAhgVADAABCAqEGAACEBEINAAAICYQaAAAQEgg1AAAgJBBqAABASCDUAACAkECoAQAAIeH/B6GjoBRGSfO9AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "beta = 0.99\n",
    "ts = np.arange(1000)\n",
    "ys = [beta**t *(1-beta) for t in ts]\n",
    "plt.plot(ts, ys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "63fecf35",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-25T11:46:57.605780Z",
     "start_time": "2024-02-25T11:46:57.411943Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_7860/1548271231.py:6: RuntimeWarning: divide by zero encountered in scalar divide\n",
      "  ys = [1./(1-beta**t) for t in ts]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fa28ed6ec80>]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAwF0lEQVR4nO3de3zU9Z3v8fdvZpLJhWQgIBkiAYOm6yXWCyiniEJXjaelWtatVdGWnu6jRxe1pOwWpbRb6tZEsctytrT04Omxbi3F3a103a5tSVuN5bAV5KKILd5SiEAaLmGSkGRmMvM9f8yFTMg9M/ObJK/n4zGPzPx+3/nl43d5NO/9/r7f788yxhgBAABkEIfdBQAAAPREQAEAABmHgAIAADIOAQUAAGQcAgoAAMg4BBQAAJBxCCgAACDjEFAAAEDGcdldwHCEw2EdPXpUBQUFsizL7nIAAMAgGGPU2tqqkpISORz9j5GMyoBy9OhRlZaW2l0GAAAYhoaGBk2fPr3fNqMyoBQUFEiK/AcWFhbaXA0AABiMlpYWlZaWxv+O92dUBpTYbZ3CwkICCgAAo8xgpmcwSRYAAGQcAgoAAMg4BBQAAJBxCCgAACDjEFAAAEDGIaAAAICMQ0ABAAAZh4ACAAAyDgEFAABknCEHlFdeeUW33nqrSkpKZFmWfvrTnyacN8ZozZo1KikpUW5urhYuXKgDBw4ktPH7/XrooYc0ZcoU5efn67bbbtMHH3wwov8QAAAwdgw5oJw5c0ZXXHGFNmzY0Ov5tWvXat26ddqwYYN27dolr9erm2++Wa2trfE2VVVV2rp1q7Zs2aLt27erra1Nn/jEJxQKhYb/XwIAAMYMyxhjhv1ly9LWrVu1ePFiSZHRk5KSElVVVenhhx+WFBktKS4u1hNPPKH77rtPPp9P5513nn74wx/qzjvvlHT26cQvvviibrnllgF/b0tLizwej3w+H8/iAQBglBjK3++kzkGpr69XY2OjKisr48fcbrcWLFigHTt2SJJ2796tYDCY0KakpEQVFRXxNj35/X61tLQkvFKhqbVTf/+zt/T4z/+QkusDAIDBSWpAaWxslCQVFxcnHC8uLo6fa2xsVHZ2tiZNmtRnm55qamrk8Xjir9LS0mSWHdfa2aXvb6/Xj149lJLrAwCAwUnJKp6ej1E2xgz4aOX+2qxatUo+ny/+amhoSFqt3WU7I90RDIVTcn0AADA4SQ0oXq9Xks4ZCWlqaoqPqni9XgUCATU3N/fZpie3263CwsKEVyq4XZHuCHQRUAAAsFNSA0pZWZm8Xq9qa2vjxwKBgOrq6jRv3jxJ0uzZs5WVlZXQ5tixY3rzzTfjbeySHQ0oYSN1MYoCAIBtXEP9Qltbm95999345/r6eu3bt09FRUWaMWOGqqqqVF1drfLycpWXl6u6ulp5eXlasmSJJMnj8eiv/uqv9Dd/8zeaPHmyioqK9Ld/+7e6/PLLddNNNyXvv2wYYgFFkgKhsFxO9rEDAMAOQw4or732mj760Y/GP69YsUKStHTpUv3gBz/QypUr1dHRoWXLlqm5uVlz587Vtm3bVFBQEP/OP/7jP8rlcunTn/60Ojo6dOONN+oHP/iBnE5nEv6Thi+7WyAJdIWVl21jMQAAjGMj2gfFLqncB2XWqv9U2Eg7v3KjphbmJPXaAACMZ7btgzIWxG7z+JkoCwCAbQgoPcRu8wSYJAsAgG0IKD1kuyLzYFhqDACAfQgoPbAXCgAA9iOg9JDljOxmyy0eAADsQ0DpIZsRFAAAbEdA6YGAAgCA/QgoPcRW8bDMGAAA+xBQeoiPoDAHBQAA2xBQeogtMw4yggIAgG0IKD2wURsAAPYjoPTAPigAANiPgNIDq3gAALAfAaUHbvEAAGA/AkoPPM0YAAD7EVB64BYPAAD2I6D0QEABAMB+BJQezs5BCdlcCQAA4xcBpQdGUAAAsB8BpYf4CAoBBQAA2xBQeuBZPAAA2I+A0gO3eAAAsB8BpYfYLR72QQEAwD4ElB4YQQEAwH4ElB6YgwIAgP0IKD3EAkqQgAIAgG0IKD24WWYMAIDtCCg9MAcFAAD7EVB6IKAAAGA/AkoPTJIFAMB+BJQe2AcFAAD7EVB64BYPAAD2I6D00P0WjzHG5moAABifCCg9uJ1OSZIxUleYgAIAgB0IKD1kuaz4e27zAABgDwJKD7FJshIBBQAAuxBQenA5HXJEB1FYagwAgD0IKL1gJQ8AAPYioPSCvVAAALAXAaUX2a7ISh5GUAAAsAcBpRfu6C2eIHNQAACwBQGlFzyPBwAAexFQehGbg8ItHgAA7EFA6QWreAAAsBcBpRexgMIqHgAA7EFA6UX8Fg9zUAAAsAUBpRfc4gEAwF4ElF4QUAAAsBcBpRdnA0rI5koAABifCCi9YA4KAAD2IqD0gn1QAACwFwGlF8xBAQDAXgSUXsT3QeEWDwAAtiCg9IIRFAAA7EVA6QVzUAAAsBcBpReMoAAAYC8CSi/c0YASZA4KAAC2IKD0Ij6CQkABAMAWSQ8oXV1d+upXv6qysjLl5uZq1qxZevTRRxUOn/1jb4zRmjVrVFJSotzcXC1cuFAHDhxIdinDxhwUAADslfSA8sQTT+h73/ueNmzYoN///vdau3atnnzySX3729+Ot1m7dq3WrVunDRs2aNeuXfJ6vbr55pvV2tqa7HKGJb7MmIACAIAtkh5Q/uu//kuf/OQntWjRIl1wwQX61Kc+pcrKSr322muSIqMn69ev1+rVq3X77beroqJCzzzzjNrb27V58+ZklzMsTJIFAMBeSQ8o8+fP169//Wu9/fbbkqTXX39d27dv18c//nFJUn19vRobG1VZWRn/jtvt1oIFC7Rjx45er+n3+9XS0pLwSiWexQMAgL1cyb7gww8/LJ/Pp4svvlhOp1OhUEiPPfaY7r77bklSY2OjJKm4uDjhe8XFxTp06FCv16ypqdE3vvGNZJfaJ0ZQAACwV9JHUJ577jk9++yz2rx5s/bs2aNnnnlG3/rWt/TMM88ktLMsK+GzMeacYzGrVq2Sz+eLvxoaGpJddgICCgAA9kr6CMqXv/xlPfLII7rrrrskSZdffrkOHTqkmpoaLV26VF6vV1JkJGXatGnx7zU1NZ0zqhLjdrvldruTXWqfuMUDAIC9kj6C0t7eLocj8bJOpzO+zLisrExer1e1tbXx84FAQHV1dZo3b16yyxkWRlAAALBX0kdQbr31Vj322GOaMWOGLrvsMu3du1fr1q3T5z//eUmRWztVVVWqrq5WeXm5ysvLVV1drby8PC1ZsiTZ5QwLAQUAAHslPaB8+9vf1te+9jUtW7ZMTU1NKikp0X333ae/+7u/i7dZuXKlOjo6tGzZMjU3N2vu3Lnatm2bCgoKkl3OsLBRGwAA9rKMMcbuIoaqpaVFHo9HPp9PhYWFSb/+B83tmv/ES8p2OfT2Nz+W9OsDADAeDeXvN8/i6UX3WzyjML8BADDqEVB64XY64++DIQIKAADpRkDpRWwERZKCLDUGACDtCCi96B5QmCgLAED6EVB64XRYcjoiu9qyWRsAAOlHQOkDS40BALAPAaUPsds8fgIKAABpR0DpA7vJAgBgHwJKH3hgIAAA9iGg9MHNCAoAALYhoPQhi0myAADYhoDSh/gclFDI5koAABh/CCh9YJIsAAD2IaD0ITZJlmXGAACkHwGlD4ygAABgHwJKH87OQSGgAACQbgSUPjCCAgCAfQgofXCzzBgAANsQUPoQG0EJcosHAIC0I6D0gVs8AADYh4DSh/gyY0ZQAABIOwJKHxhBAQDAPgSUPhBQAACwDwGlDwQUAADsQ0DpQ2wOChu1AQCQfgSUPjCCAgCAfQgofchmozYAAGxDQOkDz+IBAMA+BJQ+xAKKnxEUAADSjoDSB27xAABgHwJKH5gkCwCAfQgofWAOCgAA9iGg9MHNCAoAALYhoPQh2+mUJAUZQQEAIO0IKH1gDgoAAPYhoPSBgAIAgH0IKH2I74PCLR4AANKOgNKH7vugGGNsrgYAgPGFgNKH2AiKJAVDBBQAANKJgNIHd7eAwl4oAACkFwGlD1nObgGFibIAAKQVAaUPToclp8OSREABACDdCCj94IGBAADYg4DSj7PP4wnZXAkAAOMLAaUf8b1QGEEBACCtCCj94BYPAAD2IKD0gycaAwBgDwJKP87OQSGgAACQTgSUfvDAQAAA7EFA6UdsDkqQERQAANKKgNIPVvEAAGAPAko/uMUDAIA9CCj9iC8z5hYPAABpRUDpByMoAADYg4DSDzZqAwDAHgSUfjCCAgCAPQgo/WCjNgAA7JGSgHLkyBHde++9mjx5svLy8nTllVdq9+7d8fPGGK1Zs0YlJSXKzc3VwoULdeDAgVSUMiLc4gEAwB5JDyjNzc267rrrlJWVpZ///Od666239A//8A+aOHFivM3atWu1bt06bdiwQbt27ZLX69XNN9+s1tbWZJczIuyDAgCAPVzJvuATTzyh0tJSPf300/FjF1xwQfy9MUbr16/X6tWrdfvtt0uSnnnmGRUXF2vz5s267777kl3SsHGLBwAAeyR9BOWFF17QnDlzdMcdd2jq1Km66qqr9NRTT8XP19fXq7GxUZWVlfFjbrdbCxYs0I4dO3q9pt/vV0tLS8IrHZgkCwCAPZIeUN5//31t3LhR5eXl+uUvf6n7779fX/ziF/XP//zPkqTGxkZJUnFxccL3iouL4+d6qqmpkcfjib9KS0uTXXavmIMCAIA9kh5QwuGwrr76alVXV+uqq67Sfffdpy984QvauHFjQjvLshI+G2POORazatUq+Xy++KuhoSHZZffKzQgKAAC2SHpAmTZtmi699NKEY5dccokOHz4sSfJ6vZJ0zmhJU1PTOaMqMW63W4WFhQmvdGAOCgAA9kh6QLnuuut08ODBhGNvv/22Zs6cKUkqKyuT1+tVbW1t/HwgEFBdXZ3mzZuX7HJGJBZQggQUAADSKumreL70pS9p3rx5qq6u1qc//Wnt3LlTmzZt0qZNmyRFbu1UVVWpurpa5eXlKi8vV3V1tfLy8rRkyZJklzMi2U6nJJYZAwCQbkkPKNdcc422bt2qVatW6dFHH1VZWZnWr1+ve+65J95m5cqV6ujo0LJly9Tc3Ky5c+dq27ZtKigoSHY5I8IqHgAA7GEZY4zdRQxVS0uLPB6PfD5fSuej1L19XEv/705dOq1QLy6/PmW/BwCA8WAof795Fk8/4suMmYMCAEBaEVD6ke2KLHvmFg8AAOlFQOlHbJIsAQUAgPQioPSDfVAAALAHAaUfrOIBAMAeBJR+EFAAALAHAaUf3VfxjMLV2AAAjFoElH7ERlAk5qEAAJBOBJR+uLsHFG7zAACQNgSUfsRu8UgEFAAA0omA0g+Hw5LLEd2sjVs8AACkDQFlAKzkAQAg/QgoA4gFlCAjKAAApA0BZQCxeSh+RlAAAEgbAsoAuMUDAED6EVAGQEABACD9CCgD6L6bLAAASA8CygAYQQEAIP0IKAOIj6AQUAAASBsCygDiIyjc4gEAIG0IKAOIBRSWGQMAkD4ElAFwiwcAgPQjoAyASbIAAKQfAWUAzEEBACD9CCgDcDOCAgBA2hFQBsAcFAAA0o+AMoCcbKckqT0QsrkSAADGDwLKACblZUuSTrcHbK4EAIDxg4AygKL8SEA5eYaAAgBAuhBQBjA5GlBOEVAAAEgbAsoAiggoAACkHQFlAGdv8fhtrgQAgPGDgDKAWEDpDIbVwUoeAADSgoAygAluV3wvFEZRAABIDwLKACzLYh4KAABpRkAZBJYaAwCQXgSUQYiPoLQRUAAASAcCyiDEAkozu8kCAJAWBJRB4BYPAADpRUAZhMnc4gEAIK0IKINQNIERFAAA0omAMghFebFlxuyDAgBAOhBQBuHsJNmgzZUAADA+EFAGYXLsFk8bIygAAKQDAWUQivLdkqSWzi4FQ2GbqwEAYOwjoAyCJzdLDivyvpmJsgAApBwBZRCcDksT81jJAwBAuhBQBik+UZaAAgBAyhFQBondZAEASB8CyiDFd5MloAAAkHIElEFiBAUAgPQhoAxSUT67yQIAkC4ElEE6O0mW3WQBAEg1Asognb3FwwgKAACpRkAZpMnR3WSZJAsAQOoRUAapiFU8AACkDQFlkLo/0TgcNjZXAwDA2JbygFJTUyPLslRVVRU/ZozRmjVrVFJSotzcXC1cuFAHDhxIdSkjMik/S5IUChu1dDJRFgCAVEppQNm1a5c2bdqkD3/4wwnH165dq3Xr1mnDhg3atWuXvF6vbr75ZrW2tqaynBFxu5wqcLsksRcKAACplrKA0tbWpnvuuUdPPfWUJk2aFD9ujNH69eu1evVq3X777aqoqNAzzzyj9vZ2bd68OVXlJEXRBOahAACQDikLKA888IAWLVqkm266KeF4fX29GhsbVVlZGT/mdru1YMEC7dixo9dr+f1+tbS0JLzsEF9q3EZAAQAglVypuOiWLVu0Z88e7dq165xzjY2NkqTi4uKE48XFxTp06FCv16upqdE3vvGN5Bc6REV5jKAAAJAOSR9BaWho0PLly/Xss88qJyenz3aWZSV8Nsaccyxm1apV8vl88VdDQ0NSax6ssyt5CCgAAKRS0kdQdu/eraamJs2ePTt+LBQK6ZVXXtGGDRt08OBBSZGRlGnTpsXbNDU1nTOqEuN2u+V2u5Nd6pDF5qBwiwcAgNRK+gjKjTfeqP3792vfvn3x15w5c3TPPfdo3759mjVrlrxer2pra+PfCQQCqqur07x585JdTlJN5oGBAACkRdJHUAoKClRRUZFwLD8/X5MnT44fr6qqUnV1tcrLy1VeXq7q6mrl5eVpyZIlyS4nqYqi292zzBgAgNRKySTZgaxcuVIdHR1atmyZmpubNXfuXG3btk0FBQV2lDNoRdHN2pgkCwBAalnGmFG3b3tLS4s8Ho98Pp8KCwvT9nv3NZzW4u/8P5V4crRj1Y1p+70AAIwFQ/n7zbN4hiA2B+XkmYBGYa4DAGDUIKAMQWyZsb8rrPZAyOZqAAAYuwgoQ5CX7ZTbFeky5qEAAJA6BJQhsCzr7Hb3BBQAAFKGgDJE8d1kCSgAAKQMAWWIGEEBACD1CChDxG6yAACkHgFliNhNFgCA1COgDFF8N1keGAgAQMoQUIYoNoLS3E5AAQAgVQgoQ8QkWQAAUo+AMkSTJ8QmyRJQAABIFQLKEMVGUJiDAgBA6hBQhqgoLxJQWv1d8nfxPB4AAFKBgDJEntwsOR2WJKn5TNDmagAAGJsIKEPkcFjyFuZIkg6fare5GgAAxiYCyjCUF0+QJL3T1GpzJQAAjE0ElGG46LxIQHm3qc3mSgAAGJsIKMMQG0EhoAAAkBoElGG4aGqBJOmdPxFQAABIBQLKMFw0NTKC0tjSqdZOVvIAAJBsBJRh8ORmaWpB5Jk83OYBACD5CCjDxDwUAABSh4AyTKzkAQAgdQgow3RRcXSiLAEFAICkI6AMEyMoAACkDgFlmGJzUBqa29UR4KGBAAAkEwFlmCbnZ2tSXpaMkd47zigKAADJREAZJsuy4vuhEFAAAEguAsoIsKMsAACpQUAZgfKpTJQFACAVCCgjELvF805Tq82VAAAwthBQRiC2kuePJ9sV6ArbXA0AAGMHAWUEvIU5muB2KRQ2OnTyjN3lAAAwZhBQRsCyLF0Yv83DPBQAAJKFgDJCTJQFACD5CCgjdBEjKAAAJB0BZYRiIyjv/ImVPAAAJAsBZYTKo5u1vX/ijEJhY3M1AACMDQSUETp/Uq7cLocCXWE1nGq3uxwAAMYEAsoIOR2WLjyPeSgAACQTASUJLmIlDwAASUVASYJytrwHACCpCChJcPG0QknSvobT9hYCAMAYQUBJgmvLiuSwpPePn9HR0x12lwMAwKhHQEkCT26WriidKEna/s4Je4sBAGAMIKAkyfXl50mSXnnnuM2VAAAw+hFQkuT68imSpB3vnVSYDdsAABgRAkqSXFk6URPcLp06E9Bbx1rsLgcAgFGNgJIkWU6H/tusyZK4zQMAwEgRUJIodpuHibIAAIwMASWJYgHltT82qyMQsrkaAABGLwJKEpVNydf5E3MVCIX1av1Ju8sBAGDUIqAkkWVZmn8Rt3kAABgpAkqSXf+haEB5l4ACAMBwEVCS7LoLp8iypD80tqqppdPucgAAGJUIKEk2KT9bFSUeSYyiAAAwXEkPKDU1NbrmmmtUUFCgqVOnavHixTp48GBCG2OM1qxZo5KSEuXm5mrhwoU6cOBAskuxDcuNAQAYmaQHlLq6Oj3wwAP63e9+p9raWnV1damyslJnzpyJt1m7dq3WrVunDRs2aNeuXfJ6vbr55pvV2tqa7HJsMT8aUH777gkZw7b3AAAMlWVS/Bf0+PHjmjp1qurq6nTDDTfIGKOSkhJVVVXp4YcfliT5/X4VFxfriSee0H333TfgNVtaWuTxeOTz+VRYWJjK8ofF3xXSld+oVUcwpJ8vv16XTMu8GgEASLeh/P1O+RwUn88nSSoqKpIk1dfXq7GxUZWVlfE2brdbCxYs0I4dO1JdTlq4XU7NuzCy7f0Lrx+1uRoAAEaflAYUY4xWrFih+fPnq6KiQpLU2NgoSSouLk5oW1xcHD/Xk9/vV0tLS8Ir0336mlJJ0r/salCgK2xzNQAAjC4pDSgPPvig3njjDf34xz8+55xlWQmfjTHnHIupqamRx+OJv0pLS1NSbzLdePFUFRe6dfJMQL880HvwAgAAvUtZQHnooYf0wgsv6KWXXtL06dPjx71erySdM1rS1NR0zqhKzKpVq+Tz+eKvhoaGVJWdNC6nQ3ddM0OS9KNXD9lcDQAAo0vSA4oxRg8++KCef/55/eY3v1FZWVnC+bKyMnm9XtXW1saPBQIB1dXVad68eb1e0+12q7CwMOE1Gtx1bakclvS790/p3aY2u8sBAGDUSHpAeeCBB/Tss89q8+bNKigoUGNjoxobG9XR0SEpcmunqqpK1dXV2rp1q95880197nOfU15enpYsWZLscmw1zZOrGy+JjAptfvWwzdUAADB6JD2gbNy4UT6fTwsXLtS0adPir+eeey7eZuXKlaqqqtKyZcs0Z84cHTlyRNu2bVNBQUGyy7HdPXMjt3n+bXeDOoMhm6sBAGB0SPk+KKmQ6fugdBcOG93w5Ev6oLlD37rjCn1q9vSBvwQAwBiUUfugjHcOh6Ulc5ksCwDAUBBQ0uCO2aXKclrae/i0Dhz12V0OAAAZj4CSBucVuHXLZZHl1UyWBQBgYASUNLln7kxJ0k/2fKBjvg6bqwEAILMRUNLkv80q0jUXTFJnMKwnf3HQ7nIAAMhoBJQ0sSxLX/vEpZKk5/ce0b6G0/YWBABABiOgpNGHp0/UX14dWWb89z97S6NwhTcAAGlBQEmzlf/9z5Sb5dTuQ8362RvH7C4HAICMREBJs+LCHP31wgslSY///A/sLgsAQC8IKDb4wvWzVOLJ0ZHTHfo/v33f7nIAAMg4BBQb5GY79fDHLpYkfffl9/Snlk6bKwIAILMQUGxy2xUlumrGRLUHQvrK8/sVDjNhFgCAGAKKTSzL0jcXVyjb5dCv/9CkTdzqAQAgjoBio8tKPPr6rZG9UZ785UHtrD9lc0UAAGQGAorNllw7Q4uvLFEobPTQj/foRJvf7pIAALAdAcVmlmXpsb+4XBeel68/tfhVtWWfQsxHAQCMcwSUDJDvdmnjvbOVm+XU9ndP6Nu/ecfukgAAsBUBJUN8qLhAj/1FhSTpf/36Hf3b7g9srggAAPsQUDLI7VdP1/+47gIZI638t9f17/uO2F0SAAC2IKBkmK8tulR3XztDYSOt+JfX9eJ+ntcDABh/CCgZxuGw9NjiCn1q9nSFwkZf/PFebTvQaHdZAACkFQElAzkclp74yw9r8ZUl6gobPbB5j2rf+pPdZQEAkDYElAzldFj61h1XaNHl0xQMGf3PH76m/133noxhCTIAYOwjoGQwl9Oh9XddqbuvLZUxUs3P/6AV//K6OoMhu0sDACClCCgZLsvpUPVfXK5HP3mZnA5LW/ce0Z2bfscTkAEAYxoBZRSwLEuf/cgF+uHnr9XEvCy93nBat357u377znG7SwMAICUIKKPIvIum6IUH5utDxRPU1OrXZ76/U1/Zul9t/i67SwMAIKkIKKPMjMl5+ukD12npR2ZKkja/eli3/OMr2vHuCZsrAwAgeQgoo1Betkvf+GSFNn9hrqZPytWR0x1a8n9e1arn9/M0ZADAmEBAGcXmXThFv6i6QffMnSFJ+vHOw1r45Mv6zkvvstIHADCqWWYUbqzR0tIij8cjn8+nwsJCu8vJCK++f1Lf/M/fa/8RnyTp/Im5+vItf6bbriiRw2HZXB0AAEP7+01AGUPCYaN/f/2InvzFQR31RZYhl0+doPsXXKjbrixRlpMBMwCAfQgo41xnMKTvb6/X915+T63RFT7nT8zV/7xhlu68plQ5WU6bKwQAjEcEFEiSWjqD+tHvDuv72+vjk2cn5WXpjjmluvvaGSqbkm9zhQCA8YSAggSdwZD+dfcH+t917+mD5o748esumqx75s7UTZcUK9vF7R8AQGoRUNCrrlBYLx88rh+9ekgvv31csf/LT8zL0scvn6ZPXlGiay4oYlItACAlCCgYUMOpdm3ZdVj/8toHOt56du+UaZ4c3XpFiSovLdZVMybJSVgBACQJAQWD1hUK63fvn9K/7zuiX7zZGJ9UK0mT87N14yVTddMlxbq+/DzlZjO5FgAwfAQUDEtnMKSXDzbp52826jd/aFJr59mwku10aM4FkzS/fIquv+g8XVZSyK0gAMCQEFAwYsFQWLvqT2nbW3/Sr37/p4TJtVJkNdC1ZUW6tmyyrr2gSJdMK5CLfVYAAP0goCCpjDGqP3FG2989oVfePqHfvX/ynCcoT3C7dNWMibqqdKKunDFRV5ZOUlF+tk0VAwAyEQEFKRUMhfXGBz7trD+lnfUn9dofmxPmrsTMnJyny8/3qOJ8jy4rKdRlJR5CCwCMYwQUpFUobPT7Yy3ae7hZextOa1/Dab1//EyvbUs8Ofozb4E+5C3Qxd4C/VlxoS6cmi+3iwm4ADDWEVBgO197UG8cOa03j7TozaM+vXW0RfUneg8tDkuaUZSni6ZO0IXnTdCFUydo1pR8XTAlX5Pzs2VZTMYFgLGAgIKM1NoZ1B8aW/WHxlYdbGzRwej77quFeirIcalsSr5mTs7XzKI8zSjKU2lRnkqLcjXNk8s+LQAwihBQMGoYY9TU6td7TW1693ib3m1q03vH2/THE+066utQf/86XQ5LXk+Ozp+Yq+mT8nT+pFyVeHI0beLZnxPcrvT9xwAA+kVAwZjQGQzp8Kl2vX/8jBpOtevwqXYdOtWuhlPt+qC5XcHQwP90C9wuTS10y+vJUXFhjryFOZpa4NbUwhydV+COvC/IYRM6AEiDofz95v+9RMbKyXLqQ8UF+lBxwTnnQmGjptZOHWnu0JHTHfqgOfJq9HXomK9TR093qKWzS63+LrUe79J7fUzajcnPdmpKgVtTJrg1ZUK2Jk9wa0p+5OfkCdkqyo++8rI1MS+bhysCQIoRUDAqOR2Wpnki81Dm9NGmzd+lRl+nmlo61Rh9/cnXqaZWv463+tXU6ldTa6c6g2GdCYR05mS7Dp1sH9TvL3C7NDE/S5OigWVSXuS9JzdLE/Oir9xsFeZmyZObpcJclzy5WaxWAoBBIqBgzJrgdumiqRN00dQJfbYxxqjN36UTbQGdaIsElxNtfp1sC+jkGb9OnQnoRFtAp84E1HwmoOb2gMJGkZEZf5caTnX0ee3euF0OeXKzVJDjUmFulgpzslSYm6UJbpcKc1wqyHGpICdyPt/tUoHbpQk5Lk1wR185LuVmOVnZBGDMI6BgXLMsKxoIslQ2JX/A9uGwUUtnUCfPBHS6PaDmM0E1twd0uj3y09cR1OmOoHztQZ3uiBxv6Qiq1d8lYyR/Vzg6cuMf8Hf1xWFJ+dmRAJPvdkZ+dvucl+1SfrZTee7oz+zIsbzosbxsp3KznMqNnou9z3Y6CD4AMgYBBRgCh8PSxOhtnaEIh41a/V1q6QjK1xFUa2eXWjqj4aWzK/qKvvcH1eYPqbUzqLbOLrX5uyI/A5GQ030EJ6n/bZaUl+1STlYkuORkOZSb5VRONMDkuKLHsp1yuyLHc7IckZ+u6M8sp9zR9+4sR7Rd4s/IcQeBCEC/CChAGjgcljzR+Silw7yGMUbtgZDO+COh5Yw/pDZ/l9oDZz+3B7oibQJdavdH2rYHQmoPhtQRiLTpCIbUEYi07QiG4quhwiYyb6fnc5ZSKdsVCStulzP60xE/lh17OSPns3scS/gZfZ/lcsjtdCjLZSnb6VSW00o4l+V0RI45o+9j55xW9FzkPcEJsB8BBRglLMuK3sZxaWoSrxsMheOhpSMQCTDtgZD8wWiYiZ7r7AqrMxBSZzCkzq6QOgJhdXZFPvu7wvIHQ+oMhuOfY+38wXDkfFfkfHeBrrACXWG1Kn2haDBcjkhgcUXDjKtHgHE5ooHHYcXPuRyWXN3Ou5yWshyOXs87HWePxd5HflpyOs62cTmibaLXOtvm7O+IXcPlcMiZ8Lnb8ehnBxsbYhQhoADjXOwPb2FOVsp/lzFGgVAklPijr85gKB5U/PGfocTP0e/EX6HI+WDIxNtErhsZEQqGzn63KxxWsKvbsVBYXaHId2O19NQVNuoKh6Rgyrsk7boHGEc8yDjkdOicMONyWHJYkfYJLysSzBxWj7bRc87495Twnb7bJX6n57HBHo9d12FZcnQ/bllyWEr4fbHvWlbv37EsdTue+P3u5yxLjLilCAEFQNpYlhW9nePUubvb2MMYo1DYRIJNOKxgNPgEQ2EFQ2F1hU00DEXeB7vCCkZ/doUjbeMhKBxWV/S7XWETD0Kh8NlzXfFzJhqEosfjP3u+j34nZBSK15r4uSsUjv4Oo3D0O32JXXP407TRk8NSQpCJhSeHpeixyHsrFpSsyG3EWCCKt+0WeroHJivhvHp87tbeOre91eNnzzZWL9+J/Y7zCtx64KMX2davBBQA45oVHQ1wOaVcjY19aowxChvFg07IGIWigScU7v45HA9n4VjYibUJR45FPkeuE2mjaDAKKxRWPBBFrhlWyOjsOXPutcLdagjHvhe9TuIx0+37Z6/V/Wc4Wku42/fC3X9v9HjYRDZ3NObs74u9j52PfccYxX//YMW+ryF8ZzSYdV4+AQUAkDyR2xaS0+EUj6MavljQ6x5ewtFgY7qHox7BSNLZEGVMfPVd93AVu1bseCwoxX5nyEQCVfd2kWufDWx9Xceo27FoMIsci10/sSYTC22KfEfR707MS/1t3/7Y+k/3u9/9rp588kkdO3ZMl112mdavX6/rr7/ezpIAAJDULejJUtbYGFwbVWx7oMhzzz2nqqoqrV69Wnv37tX111+vj33sYzp8+LBdJQEAgAxh29OM586dq6uvvlobN26MH7vkkku0ePFi1dTU9PtdnmYMAMDoM5S/37aMoAQCAe3evVuVlZUJxysrK7Vjx45z2vv9frW0tCS8AADA2GVLQDlx4oRCoZCKi4sTjhcXF6uxsfGc9jU1NfJ4PPFXaelw9+IEAACjgW1zUKRzN7cxxvS64c2qVavk8/nir4aGhnSVCAAAbGDLKp4pU6bI6XSeM1rS1NR0zqiKJLndbrnd7nSVBwAAbGbLCEp2drZmz56t2trahOO1tbWaN2+eHSUBAIAMYts+KCtWrNBnPvMZzZkzRx/5yEe0adMmHT58WPfff79dJQEAgAxhW0C58847dfLkST366KM6duyYKioq9OKLL2rmzJl2lQQAADKEbfugjAT7oAAAMPpk/D4oAAAA/SGgAACAjENAAQAAGWdUPog7Nm2GLe8BABg9Yn+3BzP9dVQGlNbWVkliy3sAAEah1tZWeTyeftuMylU84XBYR48eVUFBQa9b4w9WS0uLSktL1dDQwGqgNKC/04v+Ti/6O73o7/RKVn8bY9Ta2qqSkhI5HP3PMhmVIygOh0PTp09P2vUKCwv5B55G9Hd60d/pRX+nF/2dXsno74FGTmKYJAsAADIOAQUAAGSccR1Q3G63vv71r/Ok5DShv9OL/k4v+ju96O/0sqO/R+UkWQAAMLaN6xEUAACQmQgoAAAg4xBQAABAxiGgAACAjDOuA8p3v/tdlZWVKScnR7Nnz9Zvf/tbu0sa9WpqanTNNdeooKBAU6dO1eLFi3Xw4MGENsYYrVmzRiUlJcrNzdXChQt14MABmyoeW2pqamRZlqqqquLH6O/kOnLkiO69915NnjxZeXl5uvLKK7V79+74efo7ubq6uvTVr35VZWVlys3N1axZs/Too48qHA7H29Dnw/fKK6/o1ltvVUlJiSzL0k9/+tOE84PpW7/fr4ceekhTpkxRfn6+brvtNn3wwQcjL86MU1u2bDFZWVnmqaeeMm+99ZZZvny5yc/PN4cOHbK7tFHtlltuMU8//bR58803zb59+8yiRYvMjBkzTFtbW7zN448/bgoKCsxPfvITs3//fnPnnXeaadOmmZaWFhsrH/127txpLrjgAvPhD3/YLF++PH6c/k6eU6dOmZkzZ5rPfe5z5tVXXzX19fXmV7/6lXn33Xfjbejv5PrmN79pJk+ebH72s5+Z+vp686//+q9mwoQJZv369fE29Pnwvfjii2b16tXmJz/5iZFktm7dmnB+MH17//33m/PPP9/U1taaPXv2mI9+9KPmiiuuMF1dXSOqbdwGlGuvvdbcf//9Cccuvvhi88gjj9hU0djU1NRkJJm6ujpjjDHhcNh4vV7z+OOPx9t0dnYaj8djvve979lV5qjX2tpqysvLTW1trVmwYEE8oNDfyfXwww+b+fPn93me/k6+RYsWmc9//vMJx26//XZz7733GmPo82TqGVAG07enT582WVlZZsuWLfE2R44cMQ6Hw/ziF78YUT3j8hZPIBDQ7t27VVlZmXC8srJSO3bssKmqscnn80mSioqKJEn19fVqbGxM6Hu3260FCxbQ9yPwwAMPaNGiRbrpppsSjtPfyfXCCy9ozpw5uuOOOzR16lRdddVVeuqpp+Ln6e/kmz9/vn7961/r7bffliS9/vrr2r59uz7+8Y9Los9TaTB9u3v3bgWDwYQ2JSUlqqioGHH/j8qHBY7UiRMnFAqFVFxcnHC8uLhYjY2NNlU19hhjtGLFCs2fP18VFRWSFO/f3vr+0KFDaa9xLNiyZYv27NmjXbt2nXOO/k6u999/Xxs3btSKFSv0la98RTt37tQXv/hFud1uffazn6W/U+Dhhx+Wz+fTxRdfLKfTqVAopMcee0x33323JP6Np9Jg+raxsVHZ2dmaNGnSOW1G+vd0XAaUGMuyEj4bY845huF78MEH9cYbb2j79u3nnKPvk6OhoUHLly/Xtm3blJOT02c7+js5wuGw5syZo+rqaknSVVddpQMHDmjjxo367Gc/G29HfyfPc889p2effVabN2/WZZddpn379qmqqkolJSVaunRpvB19njrD6dtk9P+4vMUzZcoUOZ3Oc9JdU1PTOUkRw/PQQw/phRde0EsvvaTp06fHj3u9Xkmi75Nk9+7dampq0uzZs+VyueRyuVRXV6d/+qd/ksvlivcp/Z0c06ZN06WXXppw7JJLLtHhw4cl8e87Fb785S/rkUce0V133aXLL79cn/nMZ/SlL31JNTU1kujzVBpM33q9XgUCATU3N/fZZrjGZUDJzs7W7NmzVVtbm3C8trZW8+bNs6mqscEYowcffFDPP/+8fvOb36isrCzhfFlZmbxeb0LfBwIB1dXV0ffDcOONN2r//v3at29f/DVnzhzdc8892rdvn2bNmkV/J9F11113zrL5t99+WzNnzpTEv+9UaG9vl8OR+KfK6XTGlxnT56kzmL6dPXu2srKyEtocO3ZMb7755sj7f0RTbEex2DLj73//++att94yVVVVJj8/3/zxj3+0u7RR7a//+q+Nx+MxL7/8sjl27Fj81d7eHm/z+OOPG4/HY55//nmzf/9+c/fdd7MkMIm6r+Ixhv5Opp07dxqXy2Uee+wx884775gf/ehHJi8vzzz77LPxNvR3ci1dutScf/758WXGzz//vJkyZYpZuXJlvA19Pnytra1m7969Zu/evUaSWbdundm7d298y43B9O39999vpk+fbn71q1+ZPXv2mD//8z9nmfFIfec73zEzZ8402dnZ5uqrr44vhcXwSer19fTTT8fbhMNh8/Wvf914vV7jdrvNDTfcYPbv329f0WNMz4BCfyfXf/zHf5iKigrjdrvNxRdfbDZt2pRwnv5OrpaWFrN8+XIzY8YMk5OTY2bNmmVWr15t/H5/vA19PnwvvfRSr/+bvXTpUmPM4Pq2o6PDPPjgg6aoqMjk5uaaT3ziE+bw4cMjrs0yxpiRjcEAAAAk17icgwIAADIbAQUAAGQcAgoAAMg4BBQAAJBxCCgAACDjEFAAAEDGIaAAAICMQ0ABAAAZh4ACAAAyDgEFAABkHAIKAADIOAQUAACQcf4/Tt8CH0mYYgMAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "beta = 0.99\n",
    "ts = np.arange(100)\n",
    "ys = [1./(1-beta**t) for t in ts]\n",
    "plt.plot(ts, ys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7b4e2dbf",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-25T02:13:27.192494Z",
     "start_time": "2024-02-25T02:13:27.180061Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"https://artemoppermann.com/wp-content/uploads/2021/08/image-56.png\" width=\"400\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Image(url='https://artemoppermann.com/wp-content/uploads/2021/08/image-56.png', width=400)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "8e7797f7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-25T02:48:08.313638Z",
     "start_time": "2024-02-25T02:48:08.304007Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"https://www.microsoft.com/en-us/research/uploads/prod/2020/02/DeepSpeed-Image-1.png\" width=\"700\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Image(url='https://www.microsoft.com/en-us/research/uploads/prod/2020/02/DeepSpeed-Image-1.png', width=700)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2589c4ba",
   "metadata": {},
   "source": [
    "## Adam => AdamW"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e5b6efe",
   "metadata": {},
   "source": [
    "- https://pytorch.org/docs/stable/generated/torch.optim.Adam.html#torch.optim.Adam\n",
    "- https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
